{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-fc061ec696c4>:62: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From <ipython-input-1-fc061ec696c4>:155: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n",
      "step 0, train_accuracy 0.1\n",
      "step 1, train_accuracy 0.06\n",
      "step 2, train_accuracy 0.2\n",
      "step 3, train_accuracy 0.1\n",
      "step 4, train_accuracy 0.06\n",
      "step 5, train_accuracy 0.12\n",
      "step 6, train_accuracy 0.12\n",
      "step 7, train_accuracy 0.18\n",
      "step 8, train_accuracy 0.3\n",
      "step 9, train_accuracy 0.28\n",
      "step 10, train_accuracy 0.54\n",
      "step 11, train_accuracy 0.36\n",
      "step 12, train_accuracy 0.48\n",
      "step 13, train_accuracy 0.4\n",
      "step 14, train_accuracy 0.48\n",
      "step 15, train_accuracy 0.6\n",
      "step 16, train_accuracy 0.6\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-fc061ec696c4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    195\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"step %d, train_accuracy %g\"\u001b[0m \u001b[1;33m%\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_accuracy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    196\u001b[0m     train_step.run(session = sess, feed_dict = {x:batch_xs, y_:batch_ys,\n\u001b[1;32m--> 197\u001b[1;33m                    keep_prob:0.5,  init_learning_rate:lr}) #神经元输出保持不变的概率 keep_prob 为0.5\n\u001b[0m\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    199\u001b[0m print(\"test accuracy %g\" %accuracy.eval(session = sess,\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, feed_dict, session)\u001b[0m\n\u001b[0;32m   2239\u001b[0m         \u001b[0mnone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mdefault\u001b[0m \u001b[0msession\u001b[0m \u001b[0mwill\u001b[0m \u001b[0mbe\u001b[0m \u001b[0mused\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2240\u001b[0m     \"\"\"\n\u001b[1;32m-> 2241\u001b[1;33m     \u001b[0m_run_using_default_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2242\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2243\u001b[0m \u001b[0m_gradient_registry\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mregistry\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mRegistry\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"gradient\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m_run_using_default_session\u001b[1;34m(operation, feed_dict, graph, session)\u001b[0m\n\u001b[0;32m   4984\u001b[0m                        \u001b[1;34m\"the operation's graph is different from the session's \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4985\u001b[0m                        \"graph.\")\n\u001b[1;32m-> 4986\u001b[1;33m   \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moperation\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4987\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4988\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    875\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    876\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 877\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    878\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    879\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1098\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1099\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1100\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1101\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1102\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1270\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1271\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1272\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1273\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1274\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1276\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1277\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1278\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1279\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1280\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1261\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1262\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1263\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1264\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1265\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1348\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[0;32m   1349\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1350\u001b[1;33m         run_metadata)\n\u001b[0m\u001b[0;32m   1351\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1352\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#coding:utf-8\n",
    "import time\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "FLAGS = None\n",
    "\"\"\"\n",
    "权重初始化\n",
    "\"\"\"\n",
    "def activation(x):\n",
    "#   return selu(x)\n",
    "    return relu(x)\n",
    "#   return tf.nn.sigmoid(x)\n",
    "#   return tf.nn.elu(x)\n",
    "#   return swish(x)\n",
    "\n",
    "def swish(x):\n",
    "    return x*tf.nn.sigmoid(x)\n",
    "\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "    return tf.truncated_normal(shape, stddev=stddev)\n",
    "\n",
    "def weight_variable(shape, stddev=0.1):\n",
    "    initial = tf.truncated_normal(shape, stddev=stddev)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.1, shape = shape)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "\"\"\"\n",
    "卷积和池化，使用卷积步长为1（stride size）,0边距（padding size）\n",
    "池化用简单传统的2x2大小的模板做max pooling\n",
    "\"\"\"\n",
    "def conv2d(x, W):\n",
    "    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding = 'SAME')\n",
    "    # tf.nn.conv2d(input, filter, strides, padding, use_cudnn_on_gpu=None, data_format=None, name=None)\n",
    "    # x(input)  : [batch, in_height, in_width, in_channels]\n",
    "    # W(filter) : [filter_height, filter_width, in_channels, out_channels]\n",
    "    # strides   : The stride of the sliding window for each dimension of input.\n",
    "    #             For the most common case of the same horizontal and vertices strides, strides = [1, stride, stride, 1]\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize = [1, 2, 2, 1],\n",
    "                          strides = [1, 2, 2, 1], padding = 'SAME')\n",
    "    # tf.nn.max_pool(value, ksize, strides, padding, data_format='NHWC', name=None)\n",
    "    # x(value)              : [batch, height, width, channels]\n",
    "    # ksize(pool大小)        : A list of ints that has length >= 4. The size of the window for each dimension of the input tensor.\n",
    "    # strides(pool滑动大小)   : A list of ints that has length >= 4. The stride of the sliding window for each dimension of the input tensor.\n",
    "\n",
    "start = time.clock() #计算开始时间\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "\n",
    "\n",
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "\n",
    "x = tf.placeholder(tf.float32,[None, 784])\n",
    "epoch_steps = tf.to_int64(tf.div(60000,tf.shape(x)[0]))\n",
    "global_step =tf.train.get_or_create_global_step()\n",
    "current_epoch =global_step//epoch_steps\n",
    "decay_times = current_epoch\n",
    "current_learning_rate = tf.multiply(init_learning_rate, tf.pow(0.575, tf.to_float(decay_times)))\n",
    "\n",
    "\n",
    "#W_1 = tf.Variable(initialize([784,L1_units_count], stddev=np.sqrt(2/784)))\n",
    "#b_1 = tf.Variable(tf.constant(0.001, shape=[L1_units_count]))\n",
    "\"\"\"\n",
    "\n",
    "第一层 卷积层\n",
    "\n",
    "x_image(batch, 28, 28, 1) -> h_pool1(batch, 14, 14, 32)\n",
    "\"\"\"\n",
    "\n",
    "x_image = tf.reshape(x, [-1, 28, 28, 1]) #最后一维代表通道数目，如果是rgb则为3 \n",
    "W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "b_conv1 = bias_variable([32])\n",
    "\n",
    "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "# x_image -> [batch, in_height, in_width, in_channels]\n",
    "#            [batch, 28, 28, 1]\n",
    "# W_conv1 -> [filter_height, filter_width, in_channels, out_channels]\n",
    "#            [5, 5, 1, 32]\n",
    "# output  -> [batch, out_height, out_width, out_channels]\n",
    "#            [batch, 28, 28, 32]\n",
    "h_pool1 = max_pool_2x2(h_conv1)\n",
    "# h_conv1 -> [batch, in_height, in_weight, in_channels]\n",
    "#            [batch, 28, 28, 32]\n",
    "# output  -> [batch, out_height, out_weight, out_channels]\n",
    "#            [batch, 14, 14, 32]\n",
    "\n",
    "\"\"\"\n",
    "第二层 卷积层\n",
    "\n",
    "h_pool1(batch, 14, 14, 32) -> h_pool2(batch, 7, 7, 64)\n",
    "\"\"\"\n",
    "W_conv2 = weight_variable([5, 5, 32, 64])\n",
    "b_conv2 = bias_variable([64])\n",
    "\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "# h_pool1 -> [batch, 14, 14, 32]\n",
    "# W_conv2 -> [5, 5, 32, 64]\n",
    "# output  -> [batch, 14, 14, 64]\n",
    "h_pool2 = max_pool_2x2(h_conv2)\n",
    "# h_conv2 -> [batch, 14, 14, 64]\n",
    "# output  -> [batch, 7, 7, 64]\n",
    "\n",
    "\"\"\"\n",
    "第三层 全连接层\n",
    "\n",
    "h_pool2(batch, 7, 7, 64) -> h_fc1(1, 1024)\n",
    "\"\"\"\n",
    "W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
    "b_fc1 = bias_variable([1024])\n",
    "\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "\"\"\"\n",
    "Dropout\n",
    "\n",
    "h_fc1 -> h_fc1_drop, 训练中启用，测试中关闭\n",
    "\"\"\"\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "\"\"\"\n",
    "第四层 Softmax输出层\n",
    "\"\"\"\n",
    "W_fc2 = weight_variable([1024, 10])\n",
    "b_fc2 = bias_variable([10])\n",
    "\n",
    "y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "\n",
    "\"\"\"\n",
    "训练和评估模型\n",
    "\n",
    "加入正则项    \n",
    "ADAM优化器来做梯度最速下降,feed_dict中加入参数keep_prob控制dropout比例\n",
    "\"\"\"\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "#cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv)) #计算交叉熵\n",
    "#train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) #使用adam优化器来以0.0001的学习率来进行微调\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "#l2_loss = tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(W_fc2)\n",
    "#total_loss= cross_entropy+4e-5*l2_loss\n",
    "total_loss= cross_entropy\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(current_learning_rate)\n",
    "gradients = optimizer.compute_gradients(total_loss)\n",
    "train_step =optimizer.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(current_learning_rate).minimize(total_loss, global_step=global_step)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1)) #判断预测标签和实际标签是否匹配\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction,\"float\"))\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "\"\"\"\n",
    "for step in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    lr = 1e-2\n",
    "    _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "        sess.run([train_step, cross_entropy, l2_loss, total_loss, current_learning_rate],\n",
    "                 feed_dict={x:batch_xs, y_:batch_ys, keep_prob:0.5, init_learning_rate:lr})\n",
    "#    if (step+1)%100 ==0 :\n",
    "    print('step %d, entropy loss:%f, l2_loss: %f, total loss:%f' %(step+1, loss, l2_loss_value, total_loss_value))\n",
    "    print(sess.run(accuracy, feed_dict={x:mnist.test.images,\n",
    "                                           y_: mnist.test.labels,keep_prob:1.0,init_learning_rate:lr}))\n",
    "    print(current_lr_value)\n",
    "    \n",
    "print(\"test accuracy %g\" %accuracy.eval(session = sess,\n",
    "      feed_dict = {x:mnist.test.images, y_:mnist.test.labels,\n",
    "                   keep_prob:1.0,  init_learning_rate:lr})) #神经元输出保持不变的概率 keep_prob 为 1，即不变，一直保持输出\n",
    "\"\"\"\n",
    "for i in range(3000): #开始训练模型，循环训练5000次\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(50) #batch大小设置为50\n",
    "    lr = 1e-2\n",
    "#    if i % 100 == 0:\n",
    "    train_accuracy = accuracy.eval(session = sess,\n",
    "                                       feed_dict = {x:batch_xs, y_:batch_ys, keep_prob:0.5, init_learning_rate:lr})\n",
    "    print(\"step %d, train_accuracy %g\" %(i, train_accuracy))\n",
    "    train_step.run(session = sess, feed_dict = {x:batch_xs, y_:batch_ys,\n",
    "                   keep_prob:0.5,  init_learning_rate:lr}) #神经元输出保持不变的概率 keep_prob 为0.5\n",
    "\n",
    "print(\"test accuracy %g\" %accuracy.eval(session = sess,\n",
    "      feed_dict = {x:mnist.test.images, y_:mnist.test.labels,\n",
    "                   keep_prob:1.0,  init_learning_rate:lr})) #神经元输出保持不变的概率 keep_prob 为 1，即不变，一直保持输出\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "end = time.clock() #计算程序结束时间\n",
    "print(\"running time is %g s\") % (end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
